{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第6步：降低学习率，调整树的数目：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath + \"RentListingInquries_FE_train.csv\")\n",
    "test = pd.read_csv(dpath + \"RentListingInquries_FE_test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据探索略过,将训练数据集和类别标签进行分割"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis =1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "准备进行交叉验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过一系列的调优，得到最优参数: bestparam= {'n_estimators':193, 'max_depth':6, 'min_child_weight':6, 'subsample': 0.7, 'colsample_bytree':0.7, 'reg_alpha':2, 'reg_lambda':1}\n",
    "这一步要降低学习率并相应提高学习器的数量，看能否得到更好的参数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    # 调用XGBoost中的DMatrix类型数据\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "    # 输入cv参数   \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "    # 导出cv结果为csv数据\n",
    "    cvresult.to_csv('1_learning_rate.csv', index_label = ['n_estimators'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb6_1 = XGBClassifier(\n",
    "        learning_rate =0.05,\n",
    "        n_estimators=2000,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=6,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.7,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alph=2,\n",
    "        reg_lambda=1,\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 执行函数训练模型并利用CV找到最佳学习率\n",
    "modelfit(xgb6_1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:2: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>test-mlogloss-mean</th>\n",
       "      <th>test-mlogloss-std</th>\n",
       "      <th>train-mlogloss-mean</th>\n",
       "      <th>train-mlogloss-std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>learning_rate</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>0.582493</td>\n",
       "      <td>0.002811</td>\n",
       "      <td>0.453548</td>\n",
       "      <td>0.000280</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>0.582485</td>\n",
       "      <td>0.002795</td>\n",
       "      <td>0.453285</td>\n",
       "      <td>0.000230</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>0.582477</td>\n",
       "      <td>0.002788</td>\n",
       "      <td>0.453071</td>\n",
       "      <td>0.000215</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>0.582469</td>\n",
       "      <td>0.002764</td>\n",
       "      <td>0.452828</td>\n",
       "      <td>0.000217</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>0.582453</td>\n",
       "      <td>0.002750</td>\n",
       "      <td>0.452651</td>\n",
       "      <td>0.000201</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               n_estimators  test-mlogloss-mean  test-mlogloss-std  \\\n",
       "learning_rate                                                        \n",
       "484                0.582493            0.002811           0.453548   \n",
       "485                0.582485            0.002795           0.453285   \n",
       "486                0.582477            0.002788           0.453071   \n",
       "487                0.582469            0.002764           0.452828   \n",
       "488                0.582453            0.002750           0.452651   \n",
       "\n",
       "               train-mlogloss-mean  train-mlogloss-std  \n",
       "learning_rate                                           \n",
       "484                       0.000280                 NaN  \n",
       "485                       0.000230                 NaN  \n",
       "486                       0.000215                 NaN  \n",
       "487                       0.000217                 NaN  \n",
       "488                       0.000201                 NaN  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取cv结果数据\n",
    "cvresult = pd.DataFrame.from_csv('1_learning_rate.csv')\n",
    "cvresult.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# n_estimators在489，学习率为0.05时，模型收敛并得到更低的-logloss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 得到最终结果并更新参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过一系列的调优，得到最终参数:\n",
    "bestparam= {'learning_rate = 0.05',\n",
    "            'n_estimators':488, \n",
    "            'max_depth':6, \n",
    "            'min_child_weight':6, \n",
    "            'subsample': 0.7, \n",
    "            'colsample_bytree':0.7, \n",
    "            'reg_alpha':2, \n",
    "            'reg_lambda':1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>74654</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2000</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>666.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74655</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3649</td>\n",
       "      <td>1824.5</td>\n",
       "      <td>1824.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74656</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2195</td>\n",
       "      <td>1097.5</td>\n",
       "      <td>2195.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74657</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1775</td>\n",
       "      <td>887.5</td>\n",
       "      <td>887.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74658</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "74654        1.0         2   2000           1000.0      666.666667       -1.0   \n",
       "74655        1.0         1   3649           1824.5     1824.500000        0.0   \n",
       "74656        1.0         0   2195           1097.5     2195.000000        1.0   \n",
       "74657        1.0         1   1775            887.5      887.500000        0.0   \n",
       "74658        1.0         2   2850           1425.0      950.000000       -1.0   \n",
       "\n",
       "       room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  \\\n",
       "74654       3.0  2016      4   16  ...         0     0      0    1       0   \n",
       "74655       2.0  2016      4    6  ...         0     0      0    0       0   \n",
       "74656       1.0  2016      4   16  ...         0     0      0    1       0   \n",
       "74657       2.0  2016      4   16  ...         0     0      0    0       0   \n",
       "74658       3.0  2016      4   26  ...         0     0      0    0       0   \n",
       "\n",
       "       water  wheelchair  wifi  windows  work  \n",
       "74654      0           0     0        0     0  \n",
       "74655      0           0     0        0     0  \n",
       "74656      0           0     0        0     0  \n",
       "74657      0           0     0        0     0  \n",
       "74658      0           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.7, gamma=0, learning_rate=0.05, max_delta_step=0,\n",
       "       max_depth=6, min_child_weight=6, missing=None, n_estimators=489,\n",
       "       n_jobs=1, nthread=None, num_class=3, objective='multi:softprob',\n",
       "       random_state=0, reg_alph=2, reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=3, silent=True, subsample=0.7)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 利用最佳参数训练模型\n",
    "xgb6_2 = XGBClassifier(\n",
    "        learning_rate =0.05,\n",
    "        n_estimators=489,  \n",
    "        max_depth=6,\n",
    "        min_child_weight=6,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.7,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        num_class= 3,\n",
    "        reg_alph=2,\n",
    "        reg_lambda=1,\n",
    "        seed=3)\n",
    "\n",
    "xgb6_2.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 利用测试集进行预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.09503317, 0.3834773 , 0.52148956],\n",
       "       [0.27732232, 0.39795902, 0.32471865],\n",
       "       [0.03913802, 0.11738072, 0.84348124],\n",
       "       ...,\n",
       "       [0.05791613, 0.29131168, 0.6507722 ],\n",
       "       [0.4461355 , 0.45378855, 0.10007603],\n",
       "       [0.03862014, 0.32186058, 0.6395193 ]], dtype=float32)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_predprob = xgb6_2.predict_proba(test)\n",
    "test_predprob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成预测结果文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = pd.DataFrame(data = test_predprob,columns= ['Interest_Level_Class_0','Interest_Level_Class_1','Interest_Level_Class_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Interest_Level_Class_0</th>\n",
       "      <th>Interest_Level_Class_1</th>\n",
       "      <th>Interest_Level_Class_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>74654</th>\n",
       "      <td>0.025235</td>\n",
       "      <td>0.172855</td>\n",
       "      <td>0.801910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74655</th>\n",
       "      <td>0.001858</td>\n",
       "      <td>0.014383</td>\n",
       "      <td>0.983759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74656</th>\n",
       "      <td>0.057916</td>\n",
       "      <td>0.291312</td>\n",
       "      <td>0.650772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74657</th>\n",
       "      <td>0.446135</td>\n",
       "      <td>0.453789</td>\n",
       "      <td>0.100076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74658</th>\n",
       "      <td>0.038620</td>\n",
       "      <td>0.321861</td>\n",
       "      <td>0.639519</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Interest_Level_Class_0  Interest_Level_Class_1  Interest_Level_Class_2\n",
       "74654                0.025235                0.172855                0.801910\n",
       "74655                0.001858                0.014383                0.983759\n",
       "74656                0.057916                0.291312                0.650772\n",
       "74657                0.446135                0.453789                0.100076\n",
       "74658                0.038620                0.321861                0.639519"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test.to_csv('y_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
